<?php

class UrlCrawler
{
	const HTTP_USER_AGENT = "\"Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.9.0.6) Gecko/2009011913 Firefox/3.0.6\"";
	
	/**
	 * @var resource
	 */
	public $ch;
	
	/**
	 * @var int
	 */
	protected $maxPages;
	
	/**
	 * @var array<SeoUrl>
	 */
	protected $foundUrls = array();
	
	/**
	 * @var array<SeoUrl>
	 */
	protected $scanUrls = array();
	
	/**
	 * @var array<SeoSitePath>
	 */
	protected $paths;
	
	/**
	 * @param SeoUrl $url
	 * @param int $maxPages
	 * @param array<SeoSitePath> $paths
	 * @return array<SeoUrl>
	 */
	public static function crawl(SeoUrl $url, $maxPages, array $paths)
	{
		Ezer_Log::debug("Crawl url [" . $url->getFullUrl() . "] max pages [$maxPages]");
		$crawler = new UrlCrawler($maxPages, $paths);
		$crawler->scan($url);
		return $crawler->getUrls();
	}
	
	/**
	 * @param string $url
	 * @param int $maxPages
	 * @param array<SeoSitePath> $paths
	 */
	protected function __construct($maxPages, array $paths)
	{
		$this->maxPages = $maxPages;
		$this->paths = $paths;
	}
	
	/**
	 * @return array<SeoUrl>
	 */
	public function getUrls()
	{
		return $this->foundUrls;
	}
	
	/**
	 * @param string $url
	 */
	public function scan(SeoUrl $url)
	{
		$this->addUrl($url);
		
		$this->initCurl($url->getFullUrl());
		$content = $this->getBody();
		$this->close();
		if(!$content)
		{
			Ezer_Log::debug("Content could not be downloaded from url [" . $url->getFullUrl() . "]");
			return false;
		}

		Ezer_Log::debug("Downloaded content [" . strlen($content) . "] from url [" . $url->getFullUrl() . "]");
		$content = strip_tags($content, "<a><frame><iframe>");
		
		$paterns = array(
			'/<a\s+[^>]*href="([^"]*)"/is',
			"/<a\s+[^>]*href='([^']*)'/is",
			'/<frame\s+[^>]*src="([^"]*)"/is',
			"/<frame\s+[^>]*src='([^']*)'/is",
			'/<iframe\s+[^>]*src="([^"]*)"/is',
			"/<iframe\s+[^>]*src='([^']*)'/is",
		);
		$subUrls = array();
		
		foreach($paterns as $patern)
		{
			if(!preg_match_all('/<a\s+[^>]*href="([^\"]*)"/is', $content, $matches))
				continue;
				
			foreach($matches[1] as $match)
				$subUrls[] = $match;
		}

		foreach($subUrls as $subUrlPath)
		{
			$subUrl = $url->getLinkUrl($subUrlPath);
			if(!$subUrl)
			{
				Ezer_Log::debug("Skip sub url [$subUrlPath]");
				continue;
			}
			
			if(!$this->canAddUrl($subUrl))
				continue;
			
			$currentUrlPath = $subUrl->getFullUrl();
			$this->initCurl($currentUrlPath);
			$headersResponse = $this->getHeader();
			
			if(!$headersResponse || $this->getError())
			{
				Ezer_Log::debug("Headers could not be downloaded from url [$currentUrlPath] error number [" . $this->getErrorNumber() . "]: " . $this->getError());
				$this->close();
				continue;
			}
			$this->close();
			
			if(!$headersResponse->isGoodCode())
			{
				Ezer_Log::debug("HTTP Error [$currentUrlPath]: $headersResponse->code: $headersResponse->codeName");
				continue;
			}
			
			if(isset($headersResponse->headers['content-type']) && is_int(strpos($headersResponse->headers['content-type'], 'text/html')))
			{
				$this->addUrl($subUrl);
			}
			elseif(isset($headersResponse->headers['content-type']))
			{
				Ezer_Log::debug("HTTP content type is [" . $headersResponse->headers['content-type'] . "]");
			}
			else
			{
				Ezer_Log::debug("HTTP has no content type");
			}				
		}
		
		while(!$this->maxPages || (count($this->scanUrls) && count($this->foundUrls) < $this->maxPages))
		{
			$nextUrl = array_shift($this->scanUrls);
			if(!$nextUrl)
				break;
				
			$this->scan($nextUrl);
		}
	}
	
	protected function canAddUrl(SeoUrl $url)
	{
		Ezer_Log::debug("Found urls [" . count($this->foundUrls) . "] max pages [$this->maxPages]");
		if($this->maxPages && count($this->foundUrls) >= $this->maxPages)
		{
			Ezer_Log::debug("Can't add sub url [" . $url->getFullUrl() . "] passed max pages [$this->maxPages]");
			return false;
		}

		if(isset($this->foundUrls[$url->getFullUrl()]))
		{
			Ezer_Log::debug("Can't add sub url [" . $url->getFullUrl() . "] url already added");
			return false;
		}
		
		if(count($this->paths) && !$url->matchesPaths($this->paths))
		{
			Ezer_Log::debug("Can't add sub url [" . $url->getFullUrl() . "] url doesn't match paths");
			return false;
		}
		
		return true;
	}
	
	protected function addUrl(SeoUrl $url)
	{
		if(isset($this->foundUrls[$url->getFullUrl()]))
			return;
			
		$this->foundUrls[$url->getFullUrl()] = $url;
		$this->scanUrls[] = $url;
	}
	
	protected function initCurl($url)
	{
		$url = trim($url);
				
		Ezer_Log::debug("Fetching url [$url]");
		$this->ch = curl_init();
		
		// set URL and other appropriate options - these can be overriden with the setOpt function if desired
		$url = self::encodeUrl($url);
		curl_setopt($this->ch, CURLOPT_URL, $url);
		curl_setopt($this->ch, CURLOPT_USERAGENT, self::HTTP_USER_AGENT);
		curl_setopt($this->ch, CURLOPT_FOLLOWLOCATION, true);
		
		curl_setopt($this->ch, CURLOPT_NOSIGNAL, true);
		curl_setopt($this->ch, CURLOPT_FORBID_REUSE, true);	
	}
	
	/**
	 * @param string $url
	 * @return string
	 */
	public static function encodeUrl($url)
	{
		return str_replace(array(' ', '[', ']'), array('%20', '%5B', '%5D'), $url);
	}
	
	/**
	 * @return false|string
	 */
	public function getError()
	{
		$err = curl_error($this->ch);
		if(!strlen($err))
			return false;
			
		return $err;
	}
	
	/**
	 * @return number
	 */
	public function getErrorNumber()
	{
		return curl_errno($this->ch);
	}
	
	/**
	 * @param int $opt
	 * @param mixed $val
	 * @return boolean
	 */
	public function setOpt($opt, $val)
	{
		return curl_setopt($this->ch, $opt, $val);
	}
	
	/**
	 * @param int $seconds
	 * @return boolean
	 */
	public function setTimeout($seconds)
	{
		return $this->setOpt(CURLOPT_TIMEOUT, $seconds);
	}
	
	/**
	 * @param int $offset
	 * @return boolean
	 */
	public function setResumeOffset($offset)
	{
		return $this->setOpt(CURLOPT_RESUME_FROM, $offset);
	}
	
	/**
	 * @return CurlHeaderResponse
	 */
	public function getHeader($noBody = false)
	{
		curl_setopt($this->ch, CURLOPT_HEADER, true);
		curl_setopt($this->ch, CURLOPT_RETURNTRANSFER, true);
		curl_setopt($this->ch, CURLOPT_BINARYTRANSFER, true);

		if($noBody)
		{
			curl_setopt($this->ch, CURLOPT_NOBODY, true);
		}
		else
		{
			curl_setopt($this->ch, CURLOPT_RANGE, '0-0');
		}
				
		$headers = curl_exec($this->ch);
		if(!$headers)
			return false;

		$headers = explode("\r\n", $headers);

		$curlHeaderResponse = new CurlHeaderResponse();
		
		$header = reset($headers);
		
		// this line is true if the protocol is HTTP (or HTTPS);
		$matches = null;
		if(preg_match('/HTTP\/?[\d.]{0,3} ([\d]{3}) ([^\n\r]+)/', $header, $matches))
		{
			$curlHeaderResponse->code = $matches[1];
			$curlHeaderResponse->codeName = $matches[2];
		}
		
		foreach ( $headers as $header )
		{
			if(!strstr($header, ':'))
			{
				if(preg_match('/HTTP\/?[\d.]{0,3} ([\d]{3}) (.+)/', $header, $matches))
				{
					$curlHeaderResponse->code = $matches[1];
					$curlHeaderResponse->codeName = $matches[2];
				}
				continue;
			}
				
			list($name, $value) = explode(':', $header, 2);
			$curlHeaderResponse->headers[trim(strtolower($name))] = trim($value);
		}
		
		if(!$noBody)
		{
			$matches = null;
			if(isset($curlHeaderResponse->headers['content-range']) && preg_match('/0-0\/([\d]+)$/', $curlHeaderResponse->headers['content-range'], $matches))
			{
				$curlHeaderResponse->headers['content-length'] = $matches[1];
			}
			else
			{
				return $this->getHeader(true);
			}
		}
		
		return $curlHeaderResponse;
	}
	
	/**
	 * @param string $destFile
	 * @return boolean
	 */
	public function getBody()
	{
		curl_setopt($this->ch, CURLOPT_HEADER, false);
		curl_setopt($this->ch, CURLOPT_NOBODY, false);
		curl_setopt($this->ch, CURLOPT_RETURNTRANSFER, true);
		
		$ret = curl_exec($this->ch);
		
		return $ret;
	}
	
	public function close()
	{
		curl_close($this->ch);
	}
}